package wox.lonice.btctrade.service.Impl;

import com.google.common.collect.Sets;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Service;
import wox.lonice.btctrade.service.WebCrawlerService;

import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.Set;

@Service
public class WebCrawlerServiceImpl implements WebCrawlerService {

    private static final String webURl = "https://www.baidu.com";

    /**
     * 新的大标头
     */
    private static Set<String> newListwebUrl = Sets.newHashSet();

    /**
     * 旧的大标头 url爬去之后放入旧的
     */
    private static Set<String> oldListwebUrl = Sets.newHashSet();

    /**
     * 图片大链接
     */
    private static Set<String> imgListwebUrl = Sets.newHashSet();

    /**
     * while循环次数限制
     */
    private static int whilrNum = 10;

    private static String webDowloadLocal = "E:\\webDownload\\Img";

    /**
     * 爬虫主程序
     *
     * @param args
     */
    public static void main(String[] args) {
        WebCrawlerServiceImpl webCrawler = new WebCrawlerServiceImpl();
        try {
            webCrawler.getMapUrl();
        } catch (IOException e) {
            e.printStackTrace();
        }

        //图片下载
        try {
            webCrawler.imgDowload();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 获取当前页面的url
     *
     * @throws IOException
     */
    private void getMapUrl() throws IOException {
        String url = webURl + "/htm/piclist1/";

        newListwebUrl.add(url + "1.htm");

        Iterator<String> it = newListwebUrl.iterator();

        int a = 0;

        while (it.hasNext()) {
            if (a > whilrNum) {
                break;
            }
            String thisUrl = it.next();

            Document documentImg = Jsoup.connect(thisUrl).userAgent("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)").get();

            //图片链接地址
            Elements elements = documentImg.select("div.layout ul.textList a");
            //分页地址
            Elements nextElements = documentImg.select("div.layout div.pageList a");

            for (int i = 0; i < nextElements.size(); i++) {
                if (i > 1 && i < 10) {
                    String nexturl = nextElements.get(i).attr("href");
                    nexturl = url + nexturl;
                    if (!oldListwebUrl.contains(nexturl)) {
                        newListwebUrl.add(nexturl);
                    }
                }
            }
            newListwebUrl.remove(thisUrl);
            oldListwebUrl.add(thisUrl);
            it = newListwebUrl.iterator();

            for (Element element : elements) {
                Elements elementSpan = element.select("span");
                // 链接中的时间 做文件夹名
                String webTime = elementSpan.get(0).text();
                String titleName = element.text();
                titleName = titleName.replace(webTime, "");
                //创建下载本地的链接
                String fileUrl = webDowloadLocal + "\\" + webTime + "\\" + titleName;

//                FileUtil.buildFile(fileUrl, true);

                String webHref = element.attr("href");

                imgListwebUrl.add(webURl + webHref + "," + fileUrl);
            }

            a++;
        }

    }

    private void imgDowload() throws IOException {

        Iterator<String> it = imgListwebUrl.iterator();

        while (it.hasNext()) {
            String imgSetString = it.next();
            String[] imgSetArray = imgSetString.split(",");
            //获取链接
            String imgHttpUrl = imgSetArray[0];
            //获取本地下载文档名称
            String fileUrl = imgSetArray[1];
            Document docImg = Jsoup.connect(imgHttpUrl).userAgent("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)").get();
            // 获取title，做文件名
            Elements elementstitle = docImg.select("title");
            System.out.println(111);
            String webImgtitle = elementstitle.text();
            webImgtitle = webImgtitle.substring(0, webImgtitle.lastIndexOf("["));
            System.out.println(webImgtitle);
            Elements elementsImg = docImg.select("div.mainAreaBlack div.picContent img");
            //下载图片使用
            for (Element elementImg : elementsImg) {
                String imgUrl = elementImg.attr("src");
                System.out.println(imgUrl);
                File file = new File(imgUrl);
                String filename = file.getName();
                String fileOneUrl = fileUrl;
                fileOneUrl = fileOneUrl + "\\" + filename;
                //需要创建文件夹
//                FileUtil.downloadNet(imgUrl, fileOneUrl);
            }
        }
    }
}
